﻿namespace Contoso.ActiveDirectory {

    #region Usings
    using System;
    using System.Collections.Concurrent;
    using System.Collections.Generic;
    using System.Configuration;
    using System.Diagnostics;
    using System.DirectoryServices;
    using System.DirectoryServices.AccountManagement;
    using System.DirectoryServices.ActiveDirectory;
    using System.Linq;
    using System.Net;
    using System.Net.NetworkInformation;
    using System.Net.Sockets;
    using System.Runtime.CompilerServices;
    using System.Runtime.InteropServices;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Threading;
    using System.Threading.Tasks;
    #endregion

    internal static class ConnectionManager {

        #region Classes
        /// <summary>
        /// Custom object for providing various DC status.  Used in the AvailableDCs collection
        /// </summary>
        /// TODO: How to ensure that the correct IP address is used?
        internal class AvailableDC {

            #region Members
            internal DateTime CacheExpiryUTC { get; private set; }
            /// <summary>
            /// Typically used only when .GetDirectoryEntry is required
            /// </summary>
            internal DomainController DC { get; set; }

            internal string DNSName { get; private set; }
            internal string IPAddress { get; set; }
            internal string DN { get; private set; }
            internal string DomainDNSName { get; private set; }
            internal string ForestDNSName { get; private set; }
            internal string SiteName { get; private set; }

            internal bool Offline {
                get {
                    if (UsePing) {
                        return (!this.PingTestOk || !this.LdapTestOk);
                    }
                    else {
                        return !this.LdapTestOk;
                    }
                }
            }
            internal bool PingTestOk { get; set; }
            internal bool LdapTestOk { get; set; }
            internal bool GCTestOk { get; set; }
            /// <summary>
            /// Read-only DC's cannot be used for updates.
            /// </summary>
            internal bool IsReadOnly { get; set; }
            /// <summary>
            /// Preferred DC establishes affinity for this DC for most operations.  Important for updates, to ensure that an update 
            /// that involves multiple transactions uses the same DC.
            /// </summary>
            internal bool IsPreferred {
                get { return isPreferred && !this.Offline; }
                set { isPreferred = value; }
            }
            private bool isPreferred;
            /// <summary>
            /// DC's to avoid, perhaps because they are slow or should not be used or preferred.
            /// </summary>
            internal bool IsLowPriority { get; set; }

            internal bool IsPDCEmulator { get; private set; }
            internal bool IsRIDRoleOwner { get; private set; }
            internal bool IsInfrastructreRoleOwner { get; private set; }
            internal bool IsSchemaMasterRoleOwner { get; private set; }
            internal bool IsDomainNamingRoleOwner { get; private set; }

            internal string OSVersion { get; private set; } 
            #endregion

            private AvailableDC() {
                this.DNSName = string.Empty;
                this.IPAddress = string.Empty;
                this.DN = string.Empty;
                this.DomainDNSName = string.Empty;
                this.ForestDNSName = string.Empty;
                this.SiteName = string.Empty;
                this.OSVersion = string.Empty;

                this.SetCacheExpiry();
            }

            internal AvailableDC(DomainController dc)
                : this() {
                this.DC = dc;
                this.DNSName = dc.Name;

                if (PreferredDCGC.Any(x => this.DNSName.StartsWith(x, StringComparison.OrdinalIgnoreCase))) {
                    this.IsPreferred = true;
                }
                if (LowPriorityDCs.Any(x => this.DNSName.StartsWith(x, StringComparison.OrdinalIgnoreCase))) {
                    this.IsLowPriority = true;
                }
            }

            internal void SetCacheExpiry() {
                this.CacheExpiryUTC = DateTime.UtcNow
                    .AddMinutes(DomainControllerCacheExpirationMinutes)
                    .AddMinutes((double)new Random(Guid.NewGuid().GetHashCode()).Next(0, 30));

                if (this.IsPreferred) {
                    // ensure that the preferred DC has a longer expiry, to avoid using non-preferred DC's
                    this.CacheExpiryUTC = this.CacheExpiryUTC.AddMinutes(30);
                }
            }

            internal void GetAdditionalInformation() {
                if (!this.Offline) {
                    try {
                        this.DN = ADDataAccess.GetDNFromLdapDN(this.DC.GetDirectoryEntry().Path);
                        this.DomainDNSName = this.DC.Domain.Name;
                        this.ForestDNSName = this.DC.Forest.Name;

                        this.OSVersion = this.DC.OSVersion;
                        this.SiteName = this.DC.SiteName;

                        foreach (ActiveDirectoryRole role in this.DC.Roles) {
                            if (role == ActiveDirectoryRole.InfrastructureRole) {
                                this.IsInfrastructreRoleOwner = true;
                            }
                            if (role == ActiveDirectoryRole.PdcRole) {
                                this.IsPDCEmulator = true;
                            }
                            if (role == ActiveDirectoryRole.RidRole) {
                                this.IsRIDRoleOwner = true;
                            }
                            if (role == ActiveDirectoryRole.SchemaRole) {
                                this.IsSchemaMasterRoleOwner = true;
                            }
                            if (role == ActiveDirectoryRole.NamingRole) {
                                this.IsDomainNamingRoleOwner = true;
                            }
                        }
                    }
                    catch { }
                }
            }            

            public override string ToString() {
                var info = new StringBuilder();

                info.AppendFormat("CacheExpiryUTC: {0}; ", this.CacheExpiryUTC.ToString("yyyy-MM-dd HH:mm"));
                info.AppendFormat("DNSName: {0}; ", this.DNSName);
                info.AppendFormat("Offline: {0}; ", this.Offline);
                info.AppendFormat("PingTestOk: {0}; ", this.PingTestOk);
                info.AppendFormat("LdapTestOk: {0}; ", this.LdapTestOk);
                info.AppendFormat("GCTestOk: {0}; ", this.GCTestOk);
                info.AppendFormat("IsReadOnly: {0}; ", this.IsReadOnly);
                info.AppendFormat("IsPreferred: {0}; ", this.IsPreferred);
                info.AppendFormat("IsLowPriority: {0}", this.IsLowPriority);

                return info.ToString();
            }
        }
        #endregion

        #region Members
        public static ConcurrentDictionary<string, ConcurrentBag<AvailableDC>> AvailableDCs { get; private set; }

        private static bool ConnectionManagerInitialized;

        /// <summary>
        /// Optional list to compensate for inaccurate results returned from Domain.FindAllDiscoverableDomainControllers(dcSiteName)
        /// Sometimes dc's from other sites are returned.  Use DCNameFilter to further refine the reults.
        /// </summary>
        private static IReadOnlyList<string> DCNameFilters { get; set; }

        private static List<string> DCSiteNames { get; set; }

        private static int DNSLookupTimeoutSeconds { get; set; }

        private static int DomainControllerCacheExpirationMinutes { get; set; }

        /// <summary>
        /// For Domain Controllers that return multiple addresses in response to System.Net.Dns.GetHostEntry,
        /// we will filter out addresses that start with these strings.
        /// </summary>
        /// <remarks>If you find yourself using this, look up the DNS Server setting: PublishAddresses
        /// http://technet.microsoft.com/en-us/library/cc959753.aspx
        /// </remarks>
        private static List<string> IPAddressFilters { get; set; }

        private static int LdapConnectionTimeoutSeconds { get; set; }

        /// <summary>
        /// Supported capabilities of domain controllers
        /// http://msdn.microsoft.com/en-us/library/cc223359%28v=prot.13%29.aspx
        /// </summary>
        private static Dictionary<string, string> LDAPSupportedCapabilities { get; set; }

        /// <summary>
        /// DC's that we prefer to not use (e.g., may be offline much of the time for business recovery testing).
        /// </summary>
        private static List<string> LowPriorityDCs { get; set; }

        private static int NetworkConnectionTimeoutSeconds { get; set; }

        private static readonly string PingData;

        /// <summary>
        /// The number of seconds to wait for all pings for a host to complete, including retries
        /// </summary>
        private static readonly int PingTestTimeoutSeconds;

        /// <summary>
        /// The number of seconds to wait for a single ping to complete
        /// </summary>
        private static readonly int PingTimeoutSeconds;

        /// <summary>
        /// Netbios name of the preferred domain controller for each domain
        /// </summary>
        internal static List<string> PreferredDCGC { get; private set; }

        private static bool RODCCheck { get; set; }

        private static int StartupTimeoutSeconds { get; set; }

        private static readonly object SyncRoot = new object();

        private static bool TestDCsAtStartup { get; set; }

        private static bool UsePing { get; set; }
        #endregion

        #region Constructors
        static ConnectionManager() {
            Debug.WriteLine(string.Format("{0}  {1}", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));

            AvailableDCs = new ConcurrentDictionary<string, ConcurrentBag<AvailableDC>>(StringComparer.OrdinalIgnoreCase);

            PreferredDCGC = ConfigurationManager.AppSettings["PreferredDCGC"] != null ? new List<string>(ConfigurationManager.AppSettings["PreferredDCGC"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) : new List<string>();
            LowPriorityDCs = ConfigurationManager.AppSettings["LowPriorityDCs"] != null ? new List<string>(ConfigurationManager.AppSettings["LowPriorityDCs"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) : new List<string>();
            DCSiteNames = ConfigurationManager.AppSettings["DCSiteNames"] != null ? new List<string>(ConfigurationManager.AppSettings["DCSiteNames"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) : new List<string>();
            DCNameFilters = ConfigurationManager.AppSettings["DCNameFilters"] != null ? new List<string>(ConfigurationManager.AppSettings["DCNameFilters"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) : new List<string>();
            DomainControllerCacheExpirationMinutes = ConfigurationManager.AppSettings["DomainControllerCacheExpirationMinutes"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["DomainControllerCacheExpirationMinutes"]) : 360;
            StartupTimeoutSeconds = ConfigurationManager.AppSettings["StartupTimeoutSeconds"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["StartupTimeoutSeconds"]) : 90;
            NetworkConnectionTimeoutSeconds = ConfigurationManager.AppSettings["NetworkConnectionTimeoutSeconds"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["NetworkConnectionTimeoutSeconds"]) : 5;
            DNSLookupTimeoutSeconds = ConfigurationManager.AppSettings["DNSLookupTimeoutSeconds"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["DNSLookupTimeoutSeconds"]) : 5;
            LdapConnectionTimeoutSeconds = ConfigurationManager.AppSettings["LdapConnectionTimeoutSeconds"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["LdapConnectionTimeoutSeconds"]) : 20;
            RODCCheck = ConfigurationManager.AppSettings["RODCCheck"] != null ? Convert.ToBoolean(ConfigurationManager.AppSettings["RODCCheck"]) : false;
            TestDCsAtStartup = ConfigurationManager.AppSettings["TestDCsAtStartup"] != null ? Convert.ToBoolean(ConfigurationManager.AppSettings["TestDCsAtStartup"]) : true;
            IPAddressFilters = ConfigurationManager.AppSettings["IPAddressFilters"] != null ? new List<string>(ConfigurationManager.AppSettings["IPAddressFilters"].Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries)) : new List<string>();

            UsePing = ConfigurationManager.AppSettings["UsePing"] != null ? Convert.ToBoolean(ConfigurationManager.AppSettings["UsePing"]) : true;
            PingData = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
            PingTimeoutSeconds = ConfigurationManager.AppSettings["PingTimeoutSeconds"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["PingTimeoutSeconds"]) : 3;
            PingTestTimeoutSeconds = ConfigurationManager.AppSettings["PingTestTimeoutSeconds"] != null ? Convert.ToInt32(ConfigurationManager.AppSettings["PingTestTimeoutSeconds"]) : 10;

            LDAPSupportedCapabilities = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.800", "ActiveDirectory");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.1791", "NTLMSignAndSeal");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.1670", "Server2003Minimum");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.1880", "LDSDigestMD5");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.1851", "LDS");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.1920", "ReadOnlyDC");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.1935", "Server2008Minimum");
            LDAPSupportedCapabilities.Add("1.2.840.113556.1.4.2080", "Server2008R2Minimum");

            // set default values
            if (DomainControllerCacheExpirationMinutes == 0) DomainControllerCacheExpirationMinutes = 360;
            if (NetworkConnectionTimeoutSeconds == 0) NetworkConnectionTimeoutSeconds = 5;
            if (DNSLookupTimeoutSeconds == 0) DNSLookupTimeoutSeconds = 5;
            if (LdapConnectionTimeoutSeconds == 0) LdapConnectionTimeoutSeconds = 20;
            if (PingTimeoutSeconds == 0) PingTimeoutSeconds = 3;
            if (PingTestTimeoutSeconds == 0) PingTestTimeoutSeconds = 10;

            Debug.WriteLine(string.Format("{0}  DomainControllerCacheExpirationMinutes: {1}", DateTime.Now.TimeOfDay, DomainControllerCacheExpirationMinutes.ToString()));
            Debug.WriteLine(string.Format("{0}  NetworkConnectionTimeoutSeconds: {1}", DateTime.Now.TimeOfDay, NetworkConnectionTimeoutSeconds.ToString()));
            Debug.WriteLine(string.Format("{0}  DNSLookupTimeoutSeconds: {1}", DateTime.Now.TimeOfDay, DNSLookupTimeoutSeconds.ToString()));
            Debug.WriteLine(string.Format("{0}  LdapConnectionTimeoutSeconds: {1}", DateTime.Now.TimeOfDay, LdapConnectionTimeoutSeconds.ToString()));
            Debug.WriteLine(string.Format("{0}  DCSiteNames: {1}", DateTime.Now.TimeOfDay, DCSiteNames.ToDelimitedString()));
            Debug.WriteLine(string.Format("{0}  DCNameFilter: {1}", DateTime.Now.TimeOfDay, DCNameFilters.ToDelimitedString()));
            Debug.WriteLine(string.Format("{0}  PreferredDCGC: {1}", DateTime.Now.TimeOfDay, PreferredDCGC.ToDelimitedString()));
        }
        #endregion

        #region Methods
        #region Initialization methods
        [MethodImpl(MethodImplOptions.NoOptimization)]
        internal static bool Initialize() {
            Debug.WriteLine(string.Format("{0}  {1}", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));

            if (!ConnectionManagerInitialized) {
                lock (SyncRoot) {
                    if (ConnectionManagerInitialized) return true;

                    if (ValidateSiteList()) {
                        var stopwatch = Stopwatch.StartNew();
                        ConnectionManagerInitialized = InitializeFilteredDCList().Result;

                        Debug.WriteLine(string.Format("Time to complete initialization: {0}", stopwatch.Elapsed));

                        #region log domain controller dictionary contents
                        Debug.WriteLine(string.Format("{0}  {1} Available domain controllers:", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));
                        foreach (var kvp in AvailableDCs) {
                            Debug.WriteLine(string.Format(" Domain: {0}", kvp.Key));
                            foreach (var availableDC in kvp.Value) {
                                if (!availableDC.Offline) {
                                    Debug.WriteLine(string.Format("  {0}", availableDC.ToString()));
                                }
                            }
                        }

                        if (TestDCsAtStartup) {
                            Debug.WriteLine(string.Format("{0}  Offline domain controllers:", DateTime.Now.TimeOfDay));
                            foreach (var availableDC in AvailableDCs.Values.SelectMany(x => x.Where(y => y.Offline == true)).ToList()) {
                                Debug.WriteLine(string.Format("  {0}", availableDC.ToString()));
                            }

                            Debug.WriteLine(string.Format("{0}  Available global catalog domain controllers:", DateTime.Now.TimeOfDay));
                            foreach (var availableDC in AvailableDCs.Values.SelectMany(x => x.Where(y => !y.Offline && y.GCTestOk)).ToList()) {
                                Debug.WriteLine(string.Format("  {0}", availableDC.DNSName));
                            }
                        }

                        Debug.WriteLine(string.Format("{0}  Preferred domain controllers:", DateTime.Now.TimeOfDay));
                        foreach (var availableDC in AvailableDCs.Values.SelectMany(x => x.Where(y => y.IsPreferred)).ToList()) {
                            Debug.WriteLine(string.Format("  {0}", availableDC.DNSName));
                        }

                        var preferredDCs = AvailableDCs.Values.SelectMany(x => x.Where(y => y.IsPreferred)).ToList();
                        if (preferredDCs.Count == 0) {
                            Debug.WriteLine(string.Format("{0}  {1} WARNING: NO PREFERRED DC'S ARE SPECIFIED!!!!", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));
                            Debug.WriteLine(string.Format("{0}  {1} WARNING: NO PREFERRED DC'S ARE SPECIFIED!!!!", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));
                            Debug.WriteLine(string.Format("{0}  {1} WARNING: NO PREFERRED DC'S ARE SPECIFIED!!!!", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));
                        }
                        #endregion
                    }
                }
            }

            return ConnectionManagerInitialized;
        }

        private static async Task<bool> InitializeFilteredDCList() {
            Debug.WriteLine(string.Format("{0}  {1} Building initial list of filtered domain controllers in all domains", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));

            bool success = false;

            await Task.Run(() => {
                try {
                    foreach (var kvp in ADDataAccess.ADDomains) {
                        Debug.WriteLine(string.Format("{0}  Creating key for domain: {1}", DateTime.Now.TimeOfDay, kvp.Key));

                        var domainDCS = new ConcurrentBag<AvailableDC>();
                        AvailableDCs.AddOrUpdate(kvp.Key, domainDCS, (TKey, TOldValue) => domainDCS);
                    }

                    using (var cts = new CancellationTokenSource()) {
                        var tasks = new List<Task>();
                        foreach (var domainName in ADDataAccess.ADDomains.Keys) {
                            tasks.Add(Task.Run(async () => {
                                await InitializeFilteredDCListThreadEntryAsync(domainName);
                            }, cts.Token));
                        }

                        Debug.WriteLine(string.Format("{0}  {1} Waiting: {2} seconds for: {3} tasks to complete.",
                            DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(), StartupTimeoutSeconds, tasks.Count));

                        if (!Task.WaitAll(tasks.ToArray(), StartupTimeoutSeconds * 1000, cts.Token)) {
                            Debug.WriteLine(string.Format("{0}  {1} WARNING: Timeout waiting for tasks to complete.", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));
                            Debug.WriteLine(string.Format("{0}  {1} WARNING: Total number of tasks: {2}; Incomplete tasks: {3}",
                                DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(), tasks.Count, tasks.Where(x => !x.IsCompleted).Count().ToString()));

                            cts.Cancel();
                        }

                        if (AvailableDCs.Any(x => x.Value.Count == 0)) {
                            foreach (var domainEntry in AvailableDCs.Where(x => x.Value.Count == 0)) {
                                Debug.WriteLine(string.Format("{0}  {1} WARNING: Domain: {2} has zero DC's.",
                                    DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(), domainEntry.Key));

                            }

                            throw new ApplicationException("One or more domains has zero DC's.");
                        }
                    }

                    #region Test the DC's for connectivity
                    if (TestDCsAtStartup) {
                        #region Log the DC's to be tested
                        Debug.WriteLine(string.Format("{0}  {1} Testing the following dc's to determine offline status:",
                            DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));

                        foreach (var dcName in AvailableDCs.Values.SelectMany(x => x.Select(y => y.DNSName)).ToList()) {
                            Debug.WriteLine(string.Format("{0}  {1}", DateTime.Now.TimeOfDay, dcName));
                        }
                        #endregion

                        Task testTask = LdapTestBindAsync(AvailableDCs.Values.SelectMany(x => x.Select(y => y)).ToList());
                        if (!testTask.Wait(TimeSpan.FromSeconds(StartupTimeoutSeconds))) {
                            Debug.WriteLine(string.Format("{0} {1} WARNING:  Timeout waiting for tests to complete.  Some DC's may be unavailable.", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName()));
                        }

                        if (AvailableDCs.Values.Any(x => !x.Any(y => !y.Offline))) {
                            success = false;
                            foreach (var domainEntry in AvailableDCs) {
                                if (!domainEntry.Value.Any(x => !x.Offline)) {
                                    Debug.WriteLine(string.Format("{0} WARNING:  Domain: {1} has no available DC's!", DateTime.Now.TimeOfDay, domainEntry.Key));
                                }
                            }
                        }
                        else if (AvailableDCs.Values.Any(x => !x.Any(y => !y.Offline && y.GCTestOk))) {
                            success = false;
                            foreach (var domainEntry in AvailableDCs) {
                                if (!domainEntry.Value.Any(x => !x.Offline && x.GCTestOk)) {
                                    Debug.WriteLine(string.Format("{0} WARNING: Domain: {1} has no available Global Catalog DC's!", DateTime.Now.TimeOfDay, domainEntry.Key));
                                }
                            }
                        }
                        else {
                            success = true;
                        }

                        // check if we have more than one preferred DC. Typically there should be one.
                        foreach (var domainEntry in AvailableDCs) {
                            if (domainEntry.Value.Where(x => !x.Offline && x.IsPreferred).Count() > 1) {
                                Debug.WriteLine(string.Format("{0} WARNING: Domain: {1} has more than one preferred DC!", DateTime.Now.TimeOfDay, domainEntry.Key));
                            }
                        }

                    } // if (TestDCsAtStartup)
                    else {
                        success = true;
                    }
                    #endregion

                    #region If we have a domain without a preferred DC, set the PDC Emulator as preferred
                    foreach (var domainEntry in AvailableDCs) {
                        if (domainEntry.Value.Count == 0) continue;
                        if (!domainEntry.Value.Any(x => !x.Offline)) continue;
                        if (!domainEntry.Value.Any(x => (!x.Offline && x.IsPreferred))) {
                            if (domainEntry.Value.Any(x => (!x.Offline && x.IsPDCEmulator))) {
                                Debug.WriteLine(string.Format("{0} INFO: Domain: {1} does not have a preferred DC specified.  Selecting PDC Emulator as preferred.", DateTime.Now.TimeOfDay, domainEntry.Key));

                                var availableDC = domainEntry.Value.Where(x => (!x.Offline && x.IsPDCEmulator)).Single();
                                availableDC.IsPreferred = true;
                            }
                            else {
                                var availableDC = domainEntry.Value.Where(x => (!x.Offline && !x.IsLowPriority)).FirstOrDefault();
                                if (availableDC == null) {
                                    availableDC = domainEntry.Value.Where(x => !x.Offline).FirstOrDefault();
                                }

                                availableDC.IsPreferred = true;
                                Debug.WriteLine(string.Format("{0} INFO: Domain: {1} does not have a preferred DC specified.  Selecting {2} as preferred.", DateTime.Now.TimeOfDay,
                                    domainEntry.Key, availableDC.DNSName));
                            }
                        }
                    }
                    #endregion
                }
                catch (Exception e) {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} {3} ", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), e.VerboseExceptionString()));
                    throw;
                }
                finally {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Finished.", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName()));
                }
            });

            return success;
        }

        /// <summary>
        /// Builds the initial list of domain controllers that will be considered available for use and tested.
        /// </summary>
        /// <param name="domainName">Netbios domain name</param>
        /// <remarks>Should complete fairly quickly.</remarks>
        private static async Task InitializeFilteredDCListThreadEntryAsync(string domainName) {
            await Task.Run(() => {
                InitializeFilteredDCListThreadEntry(domainName);
            });
        }

        private static void InitializeFilteredDCListThreadEntry(string domainName) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Building list of domain controllers for domain: {3}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), domainName));

            try {
                foreach (string dcSiteName in DCSiteNames) {
                    DomainControllerCollection domainControllers = ADDataAccess.ADDomains[domainName].FindAllDiscoverableDomainControllers(dcSiteName);

                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Domain: {3};  Site: {4}; DC's found: {5}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                        domainName, dcSiteName, domainControllers.Count));

                    foreach (DomainController dc in domainControllers) {
                        try {
                            // continue if the dc name is not on the allowed list of name prefixes
                            if (DCNameFilters.Count > 0) {
                                if (!DCNameFilters.Any(nameFilter => Regex.IsMatch(dc.Name, nameFilter, RegexOptions.IgnoreCase | RegexOptions.Compiled))) continue;
                            }
                            if (!AvailableDCs[domainName].Any(x => x.DNSName == dc.Name)) {
                                var availableDC = new AvailableDC(dc);
                                AvailableDCs[domainName].Add(availableDC);
                            }
                            else {
                                AvailableDCs[domainName].Where(x => x.DNSName == dc.Name).Single().SetCacheExpiry();
                            }
                        }
                        catch (Exception e) {
                            if (!IsDCCommunicationError(e)) {
                                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} domain: {3} error: {4}",
                                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), domainName, e.VerboseExceptionString()));
                            }
                        }
                    }
                }
            }
            catch (Exception e) {
                if (!IsDCCommunicationError(e)) {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} domain: {3} error: {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), domainName, e.VerboseExceptionString()));
                }
            }
            finally {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Finished building list of domain controllers for domain: {3}.  DC count: {4}",
                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), domainName, AvailableDCs[domainName].Count));
            }
        }

        private static bool ValidateSiteList() {
            for (int index = 0; index < DCSiteNames.Count; index++) {
                var directoryContext = new DirectoryContext(DirectoryContextType.Forest, Forest.GetCurrentForest().Name);
                ActiveDirectorySite activeDirectorySite = null;

                try {
                    activeDirectorySite = ActiveDirectorySite.FindByName(directoryContext, DCSiteNames[index]);

                    Debug.WriteLine(string.Format("{0}  {1} Found site name: {2} Location: {3} Options: {4}",
                        DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(),
                        activeDirectorySite.Name, activeDirectorySite.Location, activeDirectorySite.Options.ToString()));
                }
                catch (Exception e) {
                    Debug.WriteLine(string.Format("{0}  {1} {2}", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(), e.VerboseExceptionString()));
                }
                finally {
                    if (activeDirectorySite == null) {
                        Debug.WriteLine(string.Format("{0}  {1} Removing site name: {2} from DCSiteNames due to it is not in Forest.GetCurrentForest().Sites",
                            DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(), DCSiteNames[index]));

                        DCSiteNames.RemoveAt(index);
                    }
                }
            }

            // ensure that the local site is on the list
            try {
                ActiveDirectorySite thisSite = ActiveDirectorySite.GetComputerSite();

                if (!DCSiteNames.Any(x => thisSite.Name == x)) {
                    DCSiteNames.Add(thisSite.Name);
                    Debug.WriteLine(string.Format("{0}  {1} Adding local site name: {2} Location: {3} Options: {4}",
                        DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(),
                        thisSite.Name, thisSite.Location, thisSite.Options.ToString()));
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  {1} {2}", DateTime.Now.TimeOfDay, CustomExtensions.CurrentMethodName(), e.VerboseExceptionString()));
            }

            return DCSiteNames.Count > 0;
        }
        #endregion

        /// <summary>
        /// Helper method that uses the idiom for selecting a directory server, performing the activity,
        /// and if there is a known specific type of issue, pause, and then select another directory server.
        /// </summary>
        /// <param name="serverNameOrPath">Server name or adjusted bind path with embedded server name.  Used to add a problem
        /// serve to the Offline Server list if there is a communication error</param>
        public static TReturnType DirectoryServerHandler<TParameters, TReturnType>(
            ref TParameters parameters,
            System.Func<TParameters, TReturnType> performActivity) where TParameters : MethodParameters {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2}", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName()));

            int logonFailures = 0;
            TReturnType returnType = default(TReturnType);

            var methodName = "N/A";

            while (true) {
                try {
                    methodName = performActivity.Method.Name;
                    returnType = performActivity(parameters);
                    break;
                }
                catch (Exception e) {
                    bool? isDCCommunicationError = ConnectionManager.IsDCCommunicationError(e, ref logonFailures);
                    if (!isDCCommunicationError.HasValue) return default(TReturnType);
                    if (!isDCCommunicationError.Value) {

                        if (!ADDataAccess.Exceptions.ContainsKey(methodName)) {
                            ADDataAccess.Exceptions.Add(methodName, new List<Exception>());
                        }
                        ADDataAccess.Exceptions[methodName].Insert(0, e as Exception);

                        Debug.WriteLine(string.Format(@"{0}  [ThreadID: {1}]  {2} Exception: {3}",
                            DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                            CustomExtensions.CurrentMethodName(), e.VerboseExceptionString()));

                        throw;
                    }

                    Debug.WriteLine(string.Format(@"{0}  [ThreadID: {1}]  {2} DC: {3} Error communicating with domain controller; waiting 5 seconds.",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                        CustomExtensions.CurrentMethodName(), parameters.ServerNameOrAdjustedBindPath));

                    // mark DC as offline
                    MarkAvailableDCOffline(parameters.ServerNameOrAdjustedBindPath);
                    Thread.Sleep(TimeSpan.FromSeconds(5));
                }
            }

            return returnType;
        }

        public static bool IsDCCommunicationError<T>(T exception) where T : Exception {
            int logonFailures = 0;
            var isDCCommunicationError = IsDCCommunicationError(exception, ref logonFailures);
            if (isDCCommunicationError.HasValue) return isDCCommunicationError.Value;
            return false;
        }
        public static bool? IsDCCommunicationError<T>(T exception, ref int logonFailures) where T : Exception {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}] Error: {2}{3}", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, exception.Message, Environment.NewLine + exception.VerboseExceptionString()));

            // it's possible an object would be moved or removed on long queries or due to replication differences
            if (Regex.IsMatch(exception.Message, "There is no such object on the server", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return null;
            if (exception as System.DirectoryServices.ActiveDirectory.ActiveDirectoryObjectNotFoundException != null) return null;

            if (exception as System.DirectoryServices.ActiveDirectory.ActiveDirectoryServerDownException != null) return true;
            if (exception as PrincipalServerDownException != null) return true;

            if ((exception as DirectoryServicesCOMException != null) || (exception as COMException != null) || (exception as ActiveDirectoryOperationException != null)) {
                if (Regex.IsMatch(exception.Message, "busy", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "operation returned because the timeout period expired", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "The server does not support the requested critical extension", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "A local error has occurred", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "More data is available", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;

                // DirectoryServicesCOMException
                if (Regex.IsMatch(exception.Message, "server is not operational", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "An operations error occurred", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;

                // "The server is unwilling to process the request" usually means that there is a constraint violation, such as with the UserAccountControl and Password Policies
                //if (Regex.IsMatch(exception.Message, "The server is unwilling to process the request", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;

                if (Regex.IsMatch(exception.Message, "The directory service is unavailable", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;

                // ActiveDirectoryOperationException
                if (Regex.IsMatch(exception.Message, "Unable to obtain DNS hostname of Active Directory domain controller with ntdsa object name", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "There are no more endpoints available from the endpoint mapper", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
                if (Regex.IsMatch(exception.Message, "The remote procedure call failed and did not execute", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;

                // allow one "Logon failure: unknown user name or bad password (prevent account lockouts)"
                if (Regex.IsMatch(exception.Message, "Logon failure: unknown user name or bad password", RegexOptions.IgnoreCase | RegexOptions.Compiled) && (logonFailures++ < 1)) return true;

                if (Regex.IsMatch(exception.Message, "A device attached to the system is not functioning", RegexOptions.IgnoreCase | RegexOptions.Compiled)) return true;
            }

            return false;
        }

        private static void MarkAvailableDCOffline(string serverNameOrAdjustedBindPath) {
            AvailableDC availableDC = null;
            var domainName = ADDataAccess.GetDomainNetBiosNameFromDNOrDNSName(serverNameOrAdjustedBindPath);
            var serverName = ADDataAccess.GetServerFromLdapDN(serverNameOrAdjustedBindPath);
            if (!string.IsNullOrWhiteSpace(domainName)) {
                availableDC = AvailableDCs[domainName].Where(x => x.DNSName == serverName).FirstOrDefault();
            }
            else if (!string.IsNullOrWhiteSpace(serverName)) {
                List<AvailableDC> dcs = AvailableDCs.Values.SelectMany(x => x.Where(y => y.DNSName == serverName)).ToList();
                if (dcs.Count == 1) {
                    availableDC = dcs.Single();
                }
            }
            if (availableDC != null) {
                availableDC.PingTestOk = false;
                availableDC.LdapTestOk = false;
                availableDC.GCTestOk = false;
                availableDC.IsPreferred = false;
            }
        }

        #region DC functionality tests (LDAP, GC, RODC)
        private static bool GetReadOnlyDCStatus(string dnsName) {
            if (string.IsNullOrWhiteSpace(dnsName)) return false;
            bool isReadOnlyDC = false;

            try {
                var rootDSEPath = string.Format("LDAP://{0}/rootDSE", dnsName);
                var authenticationType = ADDataAccess.GetAdjustedAuthenticationType(rootDSEPath);

                using (var directoryEntry = new DirectoryEntry(rootDSEPath, null, null, authenticationType)) {
                    if (directoryEntry.Properties.Contains("supportedCapabilities")) {
                        for (int index = 0; index < directoryEntry.Properties["supportedCapabilities"].Count; index++) {
                            string capability = directoryEntry.Properties["supportedCapabilities"][index].ToString().Trim();
                            if (!string.IsNullOrWhiteSpace(capability) && LDAPSupportedCapabilities.ContainsKey(capability) && LDAPSupportedCapabilities[capability] == "ReadOnlyDC") {
                                isReadOnlyDC = true;
                                break;
                            }
                        }
                    }
                }
            }
            catch (Exception e) {
                if (!IsDCCommunicationError(e)) {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Exception: {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                        CustomExtensions.CurrentMethodName(), dnsName, e.VerboseExceptionString()));
                }
            }
            return isReadOnlyDC;
        }
        private static async Task LdapTestBindAsync(List<AvailableDC> availableDCs) {
            await Task.Run(() => {
                LdapTestBind(availableDCs);
            });
        }
        private static void LdapTestBind(List<AvailableDC> availableDCs) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Number of DC's to test: {3}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), availableDCs != null ? availableDCs.Count.ToString() : "NULL!"));

            try {
                using (var cts = new CancellationTokenSource()) {
                    List<Task> tasks = new List<Task>();
                    foreach (var dc in availableDCs) {
                        tasks.Add(Task.Run(() => {
                            LdapTestBind(dc, UsePing);
                        }, cts.Token));
                    }

                    if (!Task.WaitAll(tasks.ToArray(), StartupTimeoutSeconds * 1000, cts.Token)) {
                        cts.Cancel();
                        //throw new ApplicationException("Timeout waiting for threadpool threads to complete..");
                    }
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} {3} ", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), e.VerboseExceptionString()));
            }
        }
        private static void LdapTestBind(AvailableDC dc) {
            LdapTestBind(dc, usePing: true);
        }
        private static void LdapTestBind(AvailableDC dc, bool usePing) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} DC: {3}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), dc.DNSName));

            try {
                if (usePing) {
                    var ipAddress = PingHost(dc.DNSName);
                    if (ipAddress == null) {
                        dc.PingTestOk = false;
                        dc.LdapTestOk = false;
                        dc.GCTestOk = false;
                        dc.IsPreferred = false;
                        return;
                    }
                    else {
                        dc.PingTestOk = true;
                        dc.IPAddress = ipAddress.ToString();
                    }
                }

                if (LdapTestBind(dc.DNSName, 389, LdapConnectionTimeoutSeconds)) {
                    dc.LdapTestOk = true;

                    dc.GCTestOk = LdapTestBind(dc.DNSName, 3268, LdapConnectionTimeoutSeconds);

                    if (RODCCheck) {
                        dc.IsReadOnly = GetReadOnlyDCStatus(dc.DNSName);
                    }

                    dc.GetAdditionalInformation();
                }
                else {
                    dc.LdapTestOk = false;
                    dc.IsPreferred = false;
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} DC: {3} Exception: {4}",
                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                    CustomExtensions.CurrentMethodName(), dc.DNSName, e.VerboseExceptionString()));
            }
        }
        private static bool LdapTestBind(string dcName, int portNumber, int ldapTimeoutSeconds) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} host: {3} port: {4} LDAP timeout (seconds): {5}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                dcName, portNumber, ldapTimeoutSeconds));

            bool success = false;

            System.DirectoryServices.Protocols.LdapConnection ldapConnection = null;

            try {
                var task = TestNetworkConnectionAsync(dcName, portNumber);
                if (task.Wait(TimeSpan.FromSeconds(NetworkConnectionTimeoutSeconds))) {
                    ldapConnection = new System.DirectoryServices.Protocols.LdapConnection(dcName);
                    ldapConnection.Timeout = TimeSpan.FromSeconds(ldapTimeoutSeconds);
                    ldapConnection.AuthType = System.DirectoryServices.Protocols.AuthType.Kerberos;
                    ldapConnection.SessionOptions.Sealing = true;
                    ldapConnection.SessionOptions.Signing = true;
                    ldapConnection.Bind();

                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Server: {3} Reachable: {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                        ldapConnection.SessionOptions.HostName, ldapConnection.SessionOptions.HostReachable.ToString()));

                    success = ldapConnection.SessionOptions.HostReachable;
                }
                else {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} WARNING: Testing network connection to server: {3} exceeded timeout (seconds): {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                        CustomExtensions.CurrentMethodName(), dcName, NetworkConnectionTimeoutSeconds));
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Error during LDAP test bind to server: {3} {4}",
                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                    CustomExtensions.CurrentMethodName(), dcName, e.VerboseExceptionString()));

                success = false;
            }
            finally {
                if (ldapConnection != null) ldapConnection.Dispose();
            }

            return success;
        }
        #endregion

        #region Network connectivity tests
        public static IPAddress GetIpAddress(string hostName) {
            return GetIpAddress(hostName, DNSLookupTimeoutSeconds);
        }
        public static IPAddress GetIpAddress(string hostName, int timeoutSeconds) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Timeout (seconds): {4}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                hostName, timeoutSeconds));

            IPAddress ipAddress = null;
            IPHostEntry ipHostEntry = null;
            try {
                Task<IPHostEntry> task = Dns.GetHostEntryAsync(hostName);
                if (task.Wait(TimeSpan.FromSeconds(timeoutSeconds))) {
                    ipHostEntry = task.Result;
                }
                else {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} WARNING: Host: {3} Timeout (seconds) exceeded: {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                        hostName, timeoutSeconds));
                }

                if ((ipHostEntry != null) && (ipHostEntry.AddressList != null) && (ipHostEntry.AddressList.Length > 0)) {
                    if (ipHostEntry.AddressList.Length == 1) {
                        ipAddress = ipHostEntry.AddressList[0];
                        Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} IP Address: {4}",
                            DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                            hostName, ipAddress.ToString()));
                    }
                    else {
                        Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} WARNING: Host: {3} multiple IP addresses returned:",
                            DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                            hostName));

                        foreach (var address in ipHostEntry.AddressList) {
                            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} - Host: {3} Address: {4}",
                                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                                hostName, address.ToString()));
                        }

                        var filteredAddresses = new List<IPAddress>();
                        foreach (var address in ipHostEntry.AddressList) {
                            string ipAddressString = address.ToString();
                            if (!IPAddressFilters.Any(x => ipAddressString.StartsWith(x, StringComparison.OrdinalIgnoreCase))) {
                                filteredAddresses.Add(address);
                            }
                            else {
                                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Filtering out IP Address: {4}",
                                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                                    hostName, ipAddressString));
                            }
                        }

                        ipAddress = filteredAddresses.FirstOrDefault();

                        Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Selected IP Address: {4}",
                            DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                            hostName, ipAddress != null ? ipAddress.ToString() : "NULL"));
                    } // if (ipHostEntry.AddressList.Length > 1)
                } // if ((ipHostEntry != null) && (ipHostEntry.AddressList != null) && (ipHostEntry.AddressList.Length > 0))
                else {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Unable to resolve hostname to ip address: {3}", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), hostName));
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Exception: {4}",
                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                    CustomExtensions.CurrentMethodName(), hostName, e.VerboseExceptionString()));
            }

            return ipAddress;
        }
        public static async Task<IPAddress> GetIpAddressAsync(string hostName) {
            return await Task.Run<IPAddress>(() => {
                return GetIpAddress(hostName);
            });
        }
        private static IPAddress PingHost(string hostName) {
            var stopwatch = Stopwatch.StartNew();
            IPAddress ipAddress = null;

            try {
                // verify that the entry exists in DNS.  This is a workaround for extremely long timeouts caused if an entry does not exist.
                Task<IPAddress> getIpAddresstask = GetIpAddressAsync(hostName);
                if (getIpAddresstask.Wait(TimeSpan.FromSeconds(DNSLookupTimeoutSeconds))) {
                    ipAddress = getIpAddresstask.Result;
                    Task<bool> pingTask = PingHostAsync(ipAddress);
                    if (!pingTask.Wait(TimeSpan.FromSeconds(PingTestTimeoutSeconds))) {
                        Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} WARNING: Timeout pinging ip address: {3} (Host: {4}.  Elapsed time: {5}",
                            DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                            ipAddress.ToString(), hostName, stopwatch.Elapsed));

                        ipAddress = null;
                    }
                }
                else {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} WARNING: Timeout waiting to resolve hostname: {3} to ip address.  Elapsed time: {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                        hostName, stopwatch.Elapsed));
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Exception: {4}",
                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                    CustomExtensions.CurrentMethodName(), hostName, e.VerboseExceptionString()));
            }

            return ipAddress;
        }
        private static async Task<bool> PingHostAsync(IPAddress ipAddress) {
            if (ipAddress == null) {
                throw new ArgumentNullException("ipAddress");
            }

            bool pingStatus = false;
            Ping pingSender = new Ping();
            PingOptions pingOptions = new PingOptions();
            pingOptions.DontFragment = true;
            byte[] buffer = Encoding.ASCII.GetBytes(PingData);
            int attempts = 0;

            try {
                while ((attempts++ <= 2) && (!pingStatus)) {
                    var pingResult = await pingSender.SendPingAsync(ipAddress, PingTimeoutSeconds * 1000, buffer, pingOptions);
                    pingStatus = (pingResult.Status == IPStatus.Success);
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} {3} ", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), e.VerboseExceptionString()));
            }

            return pingStatus;
        }
        private static async Task<bool> TestNetworkConnectionAsync(string hostName, int portNumber) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} host: {3} port: {4}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), hostName, portNumber));

            bool success = false;

            try {
                using (System.Net.Sockets.TcpClient tcpClient = new System.Net.Sockets.TcpClient()) {
                    await tcpClient.ConnectAsync(hostName, portNumber);
                    success = tcpClient.Connected;
                }
            }
            catch (Exception e) {
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Host: {3} Exception: {4}",
                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId,
                    CustomExtensions.CurrentMethodName(), hostName, e.VerboseExceptionString()));
            }

            return success;
        }
        #endregion

        #region DC selection methods
        /// <summary>
        /// Get a list of DC's that exclude disabled or down dc's
        /// </summary>
        private static List<AvailableDC> AvailableDCsForDomain(string domainNetbiosName) {
            if (!AvailableDCs.ContainsKey(domainNetbiosName)) {
                var availableDCs = new ConcurrentBag<AvailableDC>();
                AvailableDCs.AddOrUpdate(domainNetbiosName, availableDCs, (TKey, TOldValue) => availableDCs);
            }

            List<AvailableDC> filteredDCs = AvailableDCs[domainNetbiosName]
                .Where(x => !x.Offline && (DateTime.UtcNow < x.CacheExpiryUTC))
                .ToList();

            if ((filteredDCs.Count == 0) || (!filteredDCs.Any(x => x.GCTestOk))) {
                #region Log the current status of the DC's for this domain
                foreach (var availableDC in AvailableDCs[domainNetbiosName]) {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} domain: {3} DC: {4}",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                        domainNetbiosName, availableDC.ToString()));
                }
                #endregion

                InitializeFilteredDCListThreadEntry(domainNetbiosName);
                LdapTestBind(AvailableDCs[domainNetbiosName].ToList());
                Thread.Sleep(TimeSpan.FromSeconds(1));
            }

            return filteredDCs;
        }

        /// <summary>
        /// Get a DC from the list of AvailableDC's
        /// </summary>
        internal static AvailableDC SelectDC(string domainNetBiosName, PartitionScope partitionScope = PartitionScope.LocalPartition, bool forUpdate = false) {
            Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} domain: {3} PartitionScope: {4} forUpdate: {5}",
                DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                domainNetBiosName ?? "NULL", partitionScope, forUpdate));

            #region Validation
            if (domainNetBiosName == null) {
                throw new ArgumentNullException("domainNetBiosName");
            }
            if (!ADDataAccess.ADDomains.ContainsKey(domainNetBiosName)) {
                throw new ApplicationException(string.Format("Domain information not found for netbios name: {0}", domainNetBiosName));
            } 
            #endregion

            AvailableDC availableDC = null;
            while (availableDC == null) {
                List<AvailableDC> availableDCsForDomain = AvailableDCsForDomain(domainNetBiosName);
                var filteredDCsForDomain = availableDCsForDomain;
                if ((filteredDCsForDomain != null) && (filteredDCsForDomain.Count > 0)) {
                    if (partitionScope == PartitionScope.GlobalCatalog) {
                        filteredDCsForDomain = filteredDCsForDomain
                            .Where(x => x.GCTestOk)
                            .ToList();
                    }
                    else if (forUpdate) {
                        // need a method to ensure that read-only DC's will not be used for updates.
                        filteredDCsForDomain = filteredDCsForDomain
                            .Where(x => !x.IsReadOnly)
                            .ToList();
                    }

                    // check for and remove low-priority dc's
                    if ((filteredDCsForDomain != null) && (filteredDCsForDomain.Count > 1) && filteredDCsForDomain.Where(x => !x.IsLowPriority).Count() > 0) {
                        filteredDCsForDomain = filteredDCsForDomain.Where(x => !x.IsLowPriority).ToList();
                    }

                    if ((filteredDCsForDomain != null) && (filteredDCsForDomain.Count > 0)) {
                        // check for and use a preferred dc, if available
                        if (filteredDCsForDomain.Any(x => x.IsPreferred)) {
                            availableDC = filteredDCsForDomain.Where(x => x.IsPreferred).First();
                        }
                        else {
                            availableDC = filteredDCsForDomain.FirstOrDefault();
                        }

                        if (!availableDC.IsPreferred) {
                            if (PreferredDCGC.Count > 0) {
                                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Domain controller is not a preferred DC: {3}",
                                    DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), availableDC.DNSName));
                            }
                        }
                    }
                } // if ((filteredDCs != null) && (filteredDCs.Count > 0))
                else {
                    Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} domain: {3} Warning: NO DC's in FilteredDCs list!",
                        DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(),
                        domainNetBiosName ?? "NULL"));
                }

                if (availableDC != null) break;
                Debug.WriteLine(string.Format("{0}  [ThreadID: {1}]  {2} Unable to get a dc for domain: {3}; waiting 5 seconds.", DateTime.Now.TimeOfDay, Thread.CurrentThread.ManagedThreadId, CustomExtensions.CurrentMethodName(), domainNetBiosName));
                Thread.Sleep(TimeSpan.FromSeconds(5));
            } // while (availableDC == null)

            return availableDC;
        }
        #endregion
        #endregion

    }
}
